knitr::opts_chunk$set(echo=TRUE, message=FALSE, warning=FALSE, dpi=60, out.width = "100%", eval = TRUE)
options(scipen=999)
options(knitr.kable.NA = '--') #'--'
options(knitr.kable.NAN = '--')
# load our package
library(reticulate)
Let’s make an issue to document what we are doing. We turn this chunk off because it is a one time call.
output_issue_create <- gh::gh(
endpoint = "POST /repos/{owner}/{repo}/issues",
title = "`mergin-client` error when changes made to geopackages",
body = "Seems to be occurring with programatic and `QGIS` by-hand changes to files.",
owner = params$repo_owner,
repo = params$repo_name,
assignee = params$repo_owner
)
# lets add labels
gh::gh(
endpoint = "POST /repos/{owner}/{repo}/issues/{issue_number}/labels",
labels = c("QGIS", "mergin", "documentation"),
owner = params$repo_owner,
repo = params$repo_name,
issue_number = output_issue_create$number
)
output_issue_create$number
Here is the issue number though captured from above:
output_issue_create$number
[1] 166
Ok. This is a brand new project!
I created it on the command line and pushed it to the server with:
mergin create newgraph/py_client_error --from-dir ~/Projects/gis/py_client_error
Then we want to start fresh - delete the project locally and then
download to our local machine. We could use system2 calls
to our already existing conda environment like below which
seems to work just fine as so…
# first we remove the file
system2("rm", args = c("-rf", "/Users/airvine/Projects/gis/py_client_error"))
# now we download
system2('conda', args = c('run', '-n', 'dff2', 'mergin download newgraph/py_client_error /Users/airvine/Projects/gis/py_client_error'), stdout = TRUE)
## [1] "Downloading into /Users/airvine/Projects/gis/py_client_error"
## [2] ""
## [3] "Done"
## [4] ""
reticulate to install python and set
up virtual environmentBut… We want to move to using python directly in Rstudio (helps for
story telling and reproducability) with reticulate So we
installed python fresh with:
reticulate::install_python()
And now:
reticulate::virtualenv_starter(all = TRUE)
## version path
## 1 3.12.3 /Users/airvine/.pyenv/versions/3.12.3/bin/python3.12
mergin-client package in the
virtual environmentSo do a one time move and make a py_env called
test and install the mergin-client package
reticulate::virtualenv_create("test")
reticulate::virtualenv_install("test", "mergin-client")
Now lets indicate we want to use it
reticulate::use_virtualenv("test", required = TRUE)
mergin module and create a client instancemergin module:mergin <- import("mergin")
# Create a Mergin client instance with your credentials
client <- mergin$MerginClient(login = Sys.getenv("MERGIN_USERNAME"), password = Sys.getenv("MERGIN_PASSWORD"))
system2("rm", args = c("-rf", "/Users/airvine/Projects/gis/py_client_error"))
# Download a project using the client instance
client$download_project('newgraph/py_client_error', '/Users/airvine/Projects/gis/py_client_error')
Whoa - awesome. Let’s have a look at all the available
# List attributes and methods of the module or client instance
module_methods <- py_list_attributes(mergin)
client_methods <- py_list_attributes(client)
# Print the lists
print("Methods and attributes of the module:")
## [1] "Methods and attributes of the module:"
print(module_methods)
## [1] "ClientError" "InvalidProject" "LoginError" "MerginClient" "MerginProject"
## [6] "__builtins__" "__cached__" "__doc__" "__file__" "__loader__"
## [11] "__name__" "__package__" "__path__" "__spec__" "__version__"
## [16] "client" "client_pull" "client_push" "common" "merginproject"
## [21] "utils" "version"
print("Methods and attributes of the client instance:")
## [1] "Methods and attributes of the client instance:"
print(client_methods)
## [1] "__class__" "__delattr__"
## [3] "__dict__" "__dir__"
## [5] "__doc__" "__eq__"
## [7] "__format__" "__ge__"
## [9] "__getattribute__" "__getstate__"
## [11] "__gt__" "__hash__"
## [13] "__init__" "__init_subclass__"
## [15] "__le__" "__lt__"
## [17] "__module__" "__ne__"
## [19] "__new__" "__reduce__"
## [21] "__reduce_ex__" "__repr__"
## [23] "__setattr__" "__sizeof__"
## [25] "__str__" "__subclasshook__"
## [27] "__weakref__" "_auth_params"
## [29] "_auth_session" "_check_token"
## [31] "_do_request" "_server_type"
## [33] "_server_version" "_user_info"
## [35] "add_user_permissions_to_project" "client_version"
## [37] "clone_project" "create_project"
## [39] "create_project_and_push" "create_workspace"
## [41] "default_url" "delete_project"
## [43] "delete_project_now" "download_file"
## [45] "download_file_diffs" "download_files"
## [47] "download_project" "get"
## [49] "get_file_diff" "get_projects_by_names"
## [51] "has_unfinished_pull" "has_writing_permissions"
## [53] "is_server_compatible" "log"
## [55] "login" "opener"
## [57] "paginated_projects_list" "post"
## [59] "project_file_changeset_info" "project_file_history_info"
## [61] "project_info" "project_status"
## [63] "project_user_permissions" "project_version_info"
## [65] "project_versions" "projects_list"
## [67] "pull_project" "push_project"
## [69] "remove_user_permissions_from_project" "rename_project"
## [71] "reset_local_changes" "resolve_unfinished_pull"
## [73] "server_type" "server_version"
## [75] "set_project_access" "setup_logging"
## [77] "url" "user_agent_info"
## [79] "user_info" "user_service"
## [81] "username" "workspace_service"
## [83] "workspaces_list"
Sick.
# Get help on a specific method
py_help(client$download_project)
py_help(client$project_info)
py_help(client$push_project)
Let’s get some info about the project.
project_info <- client$project_info('newgraph/py_client_error')
Lot’s here. for instance
names(project_info)
## [1] "access" "created" "creator" "disk_usage" "files" "id"
## [7] "name" "namespace" "permissions" "removed_at" "role" "tags"
## [13] "updated" "uploads" "version" "workspace_id"
processx package and mergin-client
commandline tool and current conda environment to check the
statusmergin status is not an obvious call with the python
module raw so we will use the mergin-client installed with
conda. We will probably move away from conda
soon and straight to venv so not getting into the
conda specifics. We do havetwo conda
environments installed (latest one yesterday) and they are created using
the environment.yml files in https://github.com/NewGraphEnvironment/dff-2022/ as per
the readme at https://github.com/NewGraphEnvironment/dff-2022/tree/master/scripts/qgis.
system2was really tricky to try to run outside of theworkng
directoryso we are going to try outprocessx`.
Let’s check the status of our project. First we make a function for
our processx call.
library(processx)
# Define the command and working directory
command <- "conda"
args <- c('run', '-n', 'dff2', 'mergin', 'status')
working_directory <- "/Users/airvine/Projects/gis/py_client_error"
mergin_call <- function(){
result <- tryCatch({
run(
command,
args = args,
echo = TRUE, # Print the command output live
wd = working_directory, # Set the working directory
spinner = TRUE, # Show a spinner
timeout = 60 # Timeout after 60 seconds
)
}, error = function(e) {
# Handle errors: e.g., print a custom error message
cat("An error occurred: ", e$message, "\n")
NULL # Return NULL or another appropriate value
})
# Check if the command was successful
if (!is.null(result)) {
cat("Exit status:", result$status, "\n")
cat("Output:\n", result$stdout)
} else {
cat("Failed to execute the command properly.\n")
}
}
result <- mergin_call()
## -\|/-\|/-\|/-### Server changes:
## ### Local changes:
## ### Local changes summary ###
##
## \ Exit status: 0
## Output:
## ### Server changes:
## ### Local changes:
## ### Local changes summary ###
Now let’s add a file to the project and check the status again
# Create a new file in the project directory
# to make reporoduable we will use a random number between 1:10000 appended
file_name <- paste0("/Users/airvine/Projects/gis/py_client_error/test_", sample(1:10000, 1), ".txt")
system2("touch", args = c(file_name))
result <- mergin_call()
## -\|/-\|/-\|/-### Server changes:
## ### Local changes:
##
## >>> Added:
## + test_7736.txt
## ### Local changes summary ###
##
## \| Exit status: 0
## Output:
## ### Server changes:
## ### Local changes:
##
## >>> Added:
## + test_7736.txt
## ### Local changes summary ###
Now we make a change to form_pscis.gpkg programatically
with R.
# sf::st_layers(paste0(working_directory, "/form_pscis.gpkg"))
form_in <- sf::st_read(paste0(working_directory, "/form_pscis.gpkg"))
## Reading layer `form_pscis' from data source
## `/Users/airvine/Projects/gis/py_client_error/form_pscis.gpkg' using driver `GPKG'
## Simple feature collection with 16 features and 88 fields
## Geometry type: POINT
## Dimension: XY
## Bounding box: xmin: 948064.9 ymin: 447409.8 xmax: 1036232 ymax: 1000702
## Projected CRS: NAD83 / BC Albers
# this is adding rows to the table
form_in |>
dplyr::mutate(my_crossing_reference = sample(1:10000, 1)) |>
sf::st_write(paste0(working_directory, "/form_pscis.gpkg"), "form_pscis", append = FALSE)
## Deleting layer `form_pscis' using driver `GPKG'
## Writing layer `form_pscis' to data source
## `/Users/airvine/Projects/gis/py_client_error/form_pscis.gpkg' using driver `GPKG'
## Writing 16 features with 88 fields and geometry type Point.
Now we check the status again.
# Execute the command using processx::run within tryCatch
result <- mergin_call()
## -\|/-\|/-\|/-/var/folders/mg/h910y2c54fsc99qj74dyrjph0000gn/T/tmpvz_nbhcp: line 3: 93200 Segmentation fault: 11 mergin status
##
## ERROR conda.cli.main_run:execute(124): `conda run mergin status` failed. (See above for error)
## \| An error occurred: System command 'conda' failed
## Failed to execute the command properly.
Really weird. This was all good the first time I ran it. Now it throws an error. We should look closer at the changes in that file.
Let’s push the changes to the server. Doesn’t matter if it wouldn’t let us see the status….
client$push_project('/Users/airvine/Projects/gis/py_client_error')
# client$pull_project('/Users/airvine/Projects/gis/py_client_error')
Now we check the status again.
# Execute the command using processx::run within tryCatch
result <- mergin_call()
## -\|/-\|/-\|/-### Server changes:
## ### Local changes:
## ### Local changes summary ###
##
## \| Exit status: 0
## Output:
## ### Server changes:
## ### Local changes:
## ### Local changes summary ###
That seems to clear everything up… Why? Why does it need to be pushed
for the status to work? Guessing that it is because reading in the file
and making changes in R alters the schema of
the geopackage somehow and geodiff cannot
properly track it so throws the segmentation error.
Let’s quickly visualize the differences in the file before and after
our change. This is all within R - with the table as a sf
object so is not telling us the difference in things like column types
between QGIS and R.
Hmm. Let’s just test to see if even reading it and burning it back to
file updated will cause the error. We don’t actually even
read it into an sf object
sf::st_read(paste0(working_directory, "/form_pscis.gpkg")) |>
sf::st_write(paste0(working_directory, "/form_pscis.gpkg"), "form_pscis", update = TRUE)
## Reading layer `form_pscis' from data source
## `/Users/airvine/Projects/gis/py_client_error/form_pscis.gpkg' using driver `GPKG'
## Simple feature collection with 16 features and 88 fields
## Geometry type: POINT
## Dimension: XY
## Bounding box: xmin: 948064.9 ymin: 447409.8 xmax: 1036232 ymax: 1000702
## Projected CRS: NAD83 / BC Albers
## Updating layer `form_pscis' to data source `/Users/airvine/Projects/gis/py_client_error/form_pscis.gpkg' using driver `GPKG'
## Updating existing layer form_pscis
## Writing 16 features with 88 fields and geometry type Point.
result <- mergin_call()
## -\|/-\|/-\|/-/var/folders/mg/h910y2c54fsc99qj74dyrjph0000gn/T/tmpn3qeooc4: line 3: 93219 Segmentation fault: 11 mergin status
##
## ERROR conda.cli.main_run:execute(124): `conda run mergin status` failed. (See above for error)
## \| An error occurred: System command 'conda' failed
## Failed to execute the command properly.
Ok. Looks like that is maybe the culprit. Let’s visualize the differences if we can.
geopackage file and
sf objectHere is the gpkg in QGIS.
## {
## "description":"/Users/airvine/Projects/gis/py_client_error/form_pscis.gpkg",
## "driverShortName":"GPKG",
## "driverLongName":"GeoPackage",
## "layers":[
## {
## "name":"form_pscis",
## "metadata":{},
## "geometryFields":[
## {
## "name":"geom",
## "type":"Point",
## "nullable":true,
## "extent":[
## 948064.874210938,
## 447409.81078381301,
## 1036232.08018285,
## 1000701.60915885
## ],
## "coordinateSystem":{
## "wkt":"PROJCRS[\"NAD83 / BC Albers\",\n BASEGEOGCRS[\"NAD83\",\n DATUM[\"North American Datum 1983\",\n ELLIPSOID[\"GRS 1980\",6378137,298.257222101,\n LENGTHUNIT[\"metre\",1]]],\n PRIMEM[\"Greenwich\",0,\n ANGLEUNIT[\"degree\",0.0174532925199433]],\n ID[\"EPSG\",4269]],\n CONVERSION[\"British Columbia Albers\",\n METHOD[\"Albers Equal Area\",\n ID[\"EPSG\",9822]],\n PARAMETER[\"Latitude of false origin\",45,\n ANGLEUNIT[\"degree\",0.0174532925199433],\n ID[\"EPSG\",8821]],\n PARAMETER[\"Longitude of false origin\",-126,\n ANGLEUNIT[\"degree\",0.0174532925199433],\n ID[\"EPSG\",8822]],\n PARAMETER[\"Latitude of 1st standard parallel\",50,\n ANGLEUNIT[\"degree\",0.0174532925199433],\n ID[\"EPSG\",8823]],\n PARAMETER[\"Latitude of 2nd standard parallel\",58.5,\n ANGLEUNIT[\"degree\",0.0174532925199433],\n ID[\"EPSG\",8824]],\n PARAMETER[\"Easting at false origin\",1000000,\n LENGTHUNIT[\"metre\",1],\n ID[\"EPSG\",8826]],\n PARAMETER[\"Northing at false origin\",0,\n LENGTHUNIT[\"metre\",1],\n ID[\"EPSG\",8827]]],\n CS[Cartesian,2],\n AXIS[\"(E)\",east,\n ORDER[1],\n LENGTHUNIT[\"metre\",1]],\n AXIS[\"(N)\",north,\n ORDER[2],\n LENGTHUNIT[\"metre\",1]],\n USAGE[\n SCOPE[\"Province-wide spatial data management.\"],\n AREA[\"Canada - British Columbia.\"],\n BBOX[48.25,-139.04,60.01,-114.08]],\n ID[\"EPSG\",3005]]",
## "projjson":{
## "$schema":"https://proj.org/schemas/v0.7/projjson.schema.json",
## "type":"ProjectedCRS",
## "name":"NAD83 / BC Albers",
## "base_crs":{
## "name":"NAD83",
## "datum":{
## "type":"GeodeticReferenceFrame",
## "name":"North American Datum 1983",
## "ellipsoid":{
## "name":"GRS 1980",
## "semi_major_axis":6378137,
## "inverse_flattening":298.257222101
## }
## },
## "coordinate_system":{
## "subtype":"ellipsoidal",
## "axis":[
## {
## "name":"Geodetic latitude",
## "abbreviation":"Lat",
## "direction":"north",
## "unit":"degree"
## },
## {
## "name":"Geodetic longitude",
## "abbreviation":"Lon",
## "direction":"east",
## "unit":"degree"
## }
## ]
## },
## "id":{
## "authority":"EPSG",
## "code":4269
## }
## },
## "conversion":{
## "name":"British Columbia Albers",
## "method":{
## "name":"Albers Equal Area",
## "id":{
## "authority":"EPSG",
## "code":9822
## }
## },
## "parameters":[
## {
## "name":"Latitude of false origin",
## "value":45,
## "unit":"degree",
## "id":{
## "authority":"EPSG",
## "code":8821
## }
## },
## {
## "name":"Longitude of false origin",
## "value":-126,
## "unit":"degree",
## "id":{
## "authority":"EPSG",
## "code":8822
## }
## },
## {
## "name":"Latitude of 1st standard parallel",
## "value":50,
## "unit":"degree",
## "id":{
## "authority":"EPSG",
## "code":8823
## }
## },
## {
## "name":"Latitude of 2nd standard parallel",
## "value":58.5,
## "unit":"degree",
## "id":{
## "authority":"EPSG",
## "code":8824
## }
## },
## {
## "name":"Easting at false origin",
## "value":1000000,
## "unit":"metre",
## "id":{
## "authority":"EPSG",
## "code":8826
## }
## },
## {
## "name":"Northing at false origin",
## "value":0,
## "unit":"metre",
## "id":{
## "authority":"EPSG",
## "code":8827
## }
## }
## ]
## },
## "coordinate_system":{
## "subtype":"Cartesian",
## "axis":[
## {
## "name":"Easting",
## "abbreviation":"E",
## "direction":"east",
## "unit":"metre"
## },
## {
## "name":"Northing",
## "abbreviation":"N",
## "direction":"north",
## "unit":"metre"
## }
## ]
## },
## "scope":"Province-wide spatial data management.",
## "area":"Canada - British Columbia.",
## "bbox":{
## "south_latitude":48.25,
## "west_longitude":-139.04,
## "north_latitude":60.01,
## "east_longitude":-114.08
## },
## "id":{
## "authority":"EPSG",
## "code":3005
## }
## },
## "dataAxisToSRSAxisMapping":[
## 1,
## 2
## ]
## }
## }
## ],
## "featureCount":32,
## "fidColumnName":"fid",
## "fields":[
## {
## "name":"date_time_start",
## "type":"DateTime",
## "nullable":true,
## "uniqueConstraint":false
## },
## {
## "name":"pscis_crossing_id",
## "type":"Real",
## "nullable":true,
## "uniqueConstraint":false
## },
## {
## "name":"my_crossing_reference",
## "type":"Integer",
## "nullable":true,
## "uniqueConstraint":false
## },
## {
## "name":"crew_members",
## "type":"String",
## "nullable":true,
## "uniqueConstraint":false
## },
## {
## "name":"aggregated_crossings_id",
## "type":"Real",
## "nullable":true,
## "uniqueConstraint":false
## },
## {
## "name":"camera_id",
## "type":"String",
## "nullable":true,
## "uniqueConstraint":false
## },
## {
## "name":"site_id",
## "type":"Real",
## "nullable":true,
## "uniqueConstraint":false
## },
## {
## "name":"gps_id",
## "type":"String",
## "nullable":true,
## "uniqueConstraint":false
## },
## {
## "name":"moti_chris_culvert_id",
## "type":"Integer",
## "nullable":true,
## "uniqueConstraint":false
## },
## {
## "name":"gps_waypoint_number",
## "type":"String",
## "nullable":true,
## "uniqueConstraint":false
## },
## {
## "name":"stream_name",
## "type":"String",
## "nullable":true,
## "uniqueConstraint":false
## },
## {
## "name":"road_name",
## "type":"String",
## "nullable":true,
## "uniqueConstraint":false
## },
## {
## "name":"road_km_mark",
## "type":"Real",
## "nullable":true,
## "uniqueConstraint":false
## },
## {
## "name":"road_tenure",
## "type":"String",
## "nullable":true,
## "uniqueConstraint":false
## },
## {
## "name":"crossing_type",
## "type":"String",
## "nullable":true,
## "uniqueConstraint":false
## },
## {
## "name":"crossing_subtype",
## "type":"String",
## "nullable":true,
## "uniqueConstraint":false
## },
## {
## "name":"crossing_fix",
## "type":"String",
## "nullable":true,
## "uniqueConstraint":false
## },
## {
## "name":"diameter_or_span_meters",
## "type":"Real",
## "nullable":true,
## "uniqueConstraint":false
## },
## {
## "name":"length_or_width_meters",
## "type":"Real",
## "nullable":true,
## "uniqueConstraint":false
## },
## {
## "name":"assessment_comment",
## "type":"String",
## "nullable":true,
## "uniqueConstraint":false
## },
## {
## "name":"rowid",
## "type":"Real",
## "nullable":true,
## "uniqueConstraint":false
## },
## {
## "name":"date",
## "type":"Date",
## "nullable":true,
## "uniqueConstraint":false
## },
## {
## "name":"utm_zone",
## "type":"Real",
## "nullable":true,
## "uniqueConstraint":false
## },
## {
## "name":"easting",
## "type":"Real",
## "nullable":true,
## "uniqueConstraint":false
## },
## {
## "name":"northing",
## "type":"Real",
## "nullable":true,
## "uniqueConstraint":false
## },
## {
## "name":"continuous_embeddedment_yes_no",
## "type":"String",
## "nullable":true,
## "uniqueConstraint":false
## },
## {
## "name":"average_depth_embededdment_meters",
## "type":"Real",
## "nullable":true,
## "uniqueConstraint":false
## },
## {
## "name":"resemble_channel_yes_no",
## "type":"String",
## "nullable":true,
## "uniqueConstraint":false
## },
## {
## "name":"backwatered_yes_no",
## "type":"String",
## "nullable":true,
## "uniqueConstraint":false
## },
## {
## "name":"percentage_backwatered",
## "type":"Real",
## "nullable":true,
## "uniqueConstraint":false
## },
## {
## "name":"fill_depth_meters",
## "type":"Real",
## "nullable":true,
## "uniqueConstraint":false
## },
## {
## "name":"outlet_drop_meters",
## "type":"Real",
## "nullable":true,
## "uniqueConstraint":false
## },
## {
## "name":"outlet_pool_depth_0_01m",
## "type":"Real",
## "nullable":true,
## "uniqueConstraint":false
## },
## {
## "name":"inlet_drop_yes_no",
## "type":"String",
## "nullable":true,
## "uniqueConstraint":false
## },
## {
## "name":"culvert_slope_percent",
## "type":"Real",
## "nullable":true,
## "uniqueConstraint":false
## },
## {
## "name":"downstream_channel_width_meters",
## "type":"Real",
## "nullable":true,
## "uniqueConstraint":false
## },
## {
## "name":"stream_slope",
## "type":"Real",
## "nullable":true,
## "uniqueConstraint":false
## },
## {
## "name":"beaver_activity_yes_no",
## "type":"String",
## "nullable":true,
## "uniqueConstraint":false
## },
## {
## "name":"fish_observed_yes_no",
## "type":"String",
## "nullable":true,
## "uniqueConstraint":false
## },
## {
## "name":"valley_fill",
## "type":"String",
## "nullable":true,
## "uniqueConstraint":false
## },
## {
## "name":"habitat_value",
## "type":"String",
## "nullable":true,
## "uniqueConstraint":false
## },
## {
## "name":"stream_width_ratio",
## "type":"Real",
## "nullable":true,
## "uniqueConstraint":false
## },
## {
## "name":"culvert_length_score",
## "type":"Real",
## "nullable":true,
## "uniqueConstraint":false
## },
## {
## "name":"embed_score",
## "type":"Real",
## "nullable":true,
## "uniqueConstraint":false
## },
## {
## "name":"outlet_drop_score",
## "type":"Real",
## "nullable":true,
## "uniqueConstraint":false
## },
## {
## "name":"culvert_slope_score",
## "type":"Real",
## "nullable":true,
## "uniqueConstraint":false
## },
## {
## "name":"stream_width_ratio_score",
## "type":"Real",
## "nullable":true,
## "uniqueConstraint":false
## },
## {
## "name":"final_score",
## "type":"Real",
## "nullable":true,
## "uniqueConstraint":false
## },
## {
## "name":"barrier_result",
## "type":"String",
## "nullable":true,
## "uniqueConstraint":false
## },
## {
## "name":"recommended_diameter_or_span_meters",
## "type":"Real",
## "nullable":true,
## "uniqueConstraint":false
## },
## {
## "name":"source",
## "type":"String",
## "nullable":true,
## "uniqueConstraint":false
## },
## {
## "name":"time_start",
## "type":"String",
## "nullable":true,
## "uniqueConstraint":false
## },
## {
## "name":"X.rowid",
## "type":"Real",
## "nullable":true,
## "uniqueConstraint":false
## },
## {
## "name":"mergin_user",
## "type":"String",
## "nullable":true,
## "uniqueConstraint":false
## },
## {
## "name":"erosion_issues",
## "type":"Integer",
## "nullable":true,
## "uniqueConstraint":false
## },
## {
## "name":"embankment_fill_issues",
## "type":"Integer",
## "nullable":true,
## "uniqueConstraint":false
## },
## {
## "name":"blockage_issues",
## "type":"Integer",
## "nullable":true,
## "uniqueConstraint":false
## },
## {
## "name":"condition_rank",
## "type":"Integer",
## "nullable":true,
## "uniqueConstraint":false
## },
## {
## "name":"condition_notes",
## "type":"String",
## "nullable":true,
## "uniqueConstraint":false
## },
## {
## "name":"likelihood_flood_event_affecting_culvert",
## "type":"Integer",
## "nullable":true,
## "uniqueConstraint":false
## },
## {
## "name":"consequence_flood_event_affecting_culvert",
## "type":"Integer",
## "nullable":true,
## "uniqueConstraint":false
## },
## {
## "name":"climate_change_flood_risk",
## "type":"Integer",
## "nullable":true,
## "uniqueConstraint":false
## },
## {
## "name":"vulnerability_rank",
## "type":"Integer",
## "nullable":true,
## "uniqueConstraint":false
## },
## {
## "name":"climate_notes",
## "type":"String",
## "nullable":true,
## "uniqueConstraint":false
## },
## {
## "name":"traffic_volume",
## "type":"Integer",
## "nullable":true,
## "uniqueConstraint":false
## },
## {
## "name":"community_access",
## "type":"Integer",
## "nullable":true,
## "uniqueConstraint":false
## },
## {
## "name":"cost",
## "type":"Integer",
## "nullable":true,
## "uniqueConstraint":false
## },
## {
## "name":"constructability",
## "type":"Integer",
## "nullable":true,
## "uniqueConstraint":false
## },
## {
## "name":"fish_bearing",
## "type":"Integer",
## "nullable":true,
## "uniqueConstraint":false
## },
## {
## "name":"environmental_impacts",
## "type":"Integer",
## "nullable":true,
## "uniqueConstraint":false
## },
## {
## "name":"priority_rank",
## "type":"Integer",
## "nullable":true,
## "uniqueConstraint":false
## },
## {
## "name":"overall_rank",
## "type":"Integer",
## "nullable":true,
## "uniqueConstraint":false
## },
## {
## "name":"priority_notes",
## "type":"String",
## "nullable":true,
## "uniqueConstraint":false
## },
## {
## "name":"photo_road",
## "type":"String",
## "nullable":true,
## "uniqueConstraint":false
## },
## {
## "name":"photo_upstream",
## "type":"String",
## "nullable":true,
## "uniqueConstraint":false
## },
## {
## "name":"photo_downstream",
## "type":"String",
## "nullable":true,
## "uniqueConstraint":false
## },
## {
## "name":"photo_inlet",
## "type":"String",
## "nullable":true,
## "uniqueConstraint":false
## },
## {
## "name":"photo_barrel",
## "type":"String",
## "nullable":true,
## "uniqueConstraint":false
## },
## {
## "name":"photo_outlet",
## "type":"String",
## "nullable":true,
## "uniqueConstraint":false
## },
## {
## "name":"photo_condition",
## "type":"String",
## "nullable":true,
## "uniqueConstraint":false
## },
## {
## "name":"photo_embankment_fill",
## "type":"String",
## "nullable":true,
## "uniqueConstraint":false
## },
## {
## "name":"photo_blockage",
## "type":"String",
## "nullable":true,
## "uniqueConstraint":false
## },
## {
## "name":"photo_paper_card",
## "type":"String",
## "nullable":true,
## "uniqueConstraint":false
## },
## {
## "name":"photo_extra1",
## "type":"String",
## "nullable":true,
## "uniqueConstraint":false
## },
## {
## "name":"photo_extra2",
## "type":"String",
## "nullable":true,
## "uniqueConstraint":false
## },
## {
## "name":"photo_extra1_tag",
## "type":"String",
## "nullable":true,
## "uniqueConstraint":false
## },
## {
## "name":"photo_extra2_tag",
## "type":"String",
## "nullable":true,
## "uniqueConstraint":false
## },
## {
## "name":"link_method_phase1",
## "type":"String",
## "nullable":true,
## "uniqueConstraint":false
## }
## ]
## }
## ],
## "metadata":{},
## "domains":{},
## "relationships":{}
## }
| name | type |
|---|---|
| date_time_start | DateTime |
| pscis_crossing_id | Real |
| my_crossing_reference | Integer |
| crew_members | String |
| aggregated_crossings_id | Real |
| camera_id | String |
| site_id | Real |
| gps_id | String |
| moti_chris_culvert_id | Integer |
| gps_waypoint_number | String |
| stream_name | String |
| road_name | String |
| road_km_mark | Real |
| road_tenure | String |
| crossing_type | String |
| crossing_subtype | String |
| crossing_fix | String |
| diameter_or_span_meters | Real |
| length_or_width_meters | Real |
| assessment_comment | String |
| rowid | Real |
| date | Date |
| utm_zone | Real |
| easting | Real |
| northing | Real |
| continuous_embeddedment_yes_no | String |
| average_depth_embededdment_meters | Real |
| resemble_channel_yes_no | String |
| backwatered_yes_no | String |
| percentage_backwatered | Real |
| fill_depth_meters | Real |
| outlet_drop_meters | Real |
| outlet_pool_depth_0_01m | Real |
| inlet_drop_yes_no | String |
| culvert_slope_percent | Real |
| downstream_channel_width_meters | Real |
| stream_slope | Real |
| beaver_activity_yes_no | String |
| fish_observed_yes_no | String |
| valley_fill | String |
| habitat_value | String |
| stream_width_ratio | Real |
| culvert_length_score | Real |
| embed_score | Real |
| outlet_drop_score | Real |
| culvert_slope_score | Real |
| stream_width_ratio_score | Real |
| final_score | Real |
| barrier_result | String |
| recommended_diameter_or_span_meters | Real |
| source | String |
| time_start | String |
| X.rowid | Real |
| mergin_user | String |
| erosion_issues | Integer |
| embankment_fill_issues | Integer |
| blockage_issues | Integer |
| condition_rank | Integer |
| condition_notes | String |
| likelihood_flood_event_affecting_culvert | Integer |
| consequence_flood_event_affecting_culvert | Integer |
| climate_change_flood_risk | Integer |
| vulnerability_rank | Integer |
| climate_notes | String |
| traffic_volume | Integer |
| community_access | Integer |
| cost | Integer |
| constructability | Integer |
| fish_bearing | Integer |
| environmental_impacts | Integer |
| priority_rank | Integer |
| overall_rank | Integer |
| priority_notes | String |
| photo_road | String |
| photo_upstream | String |
| photo_downstream | String |
| photo_inlet | String |
| photo_barrel | String |
| photo_outlet | String |
| photo_condition | String |
| photo_embankment_fill | String |
| photo_blockage | String |
| photo_paper_card | String |
| photo_extra1 | String |
| photo_extra2 | String |
| photo_extra1_tag | String |
| photo_extra2_tag | String |
| link_method_phase1 | String |
Here is gpkg columns and types in R
gpkp_r <- sf::st_read(paste0(working_directory, "/form_pscis.gpkg"))
## Reading layer `form_pscis' from data source
## `/Users/airvine/Projects/gis/py_client_error/form_pscis.gpkg' using driver `GPKG'
## Simple feature collection with 32 features and 88 fields
## Geometry type: POINT
## Dimension: XY
## Bounding box: xmin: 948064.9 ymin: 447409.8 xmax: 1036232 ymax: 1000702
## Projected CRS: NAD83 / BC Albers
# maek tibble of colun names and class
gpkp_r_sum <- purrr::map2_dfr(
names(gpkp_r),
sapply(gpkp_r, function(col) class(col)[1]),
~tibble::tibble(name = .x, type = .y)
)
fpr::fpr_kable(gpkp_r_sum, font = params$font_set)
| name | type |
|---|---|
| date_time_start | POSIXct |
| pscis_crossing_id | numeric |
| my_crossing_reference | integer |
| crew_members | character |
| aggregated_crossings_id | numeric |
| camera_id | character |
| site_id | numeric |
| gps_id | character |
| moti_chris_culvert_id | integer |
| gps_waypoint_number | character |
| stream_name | character |
| road_name | character |
| road_km_mark | numeric |
| road_tenure | character |
| crossing_type | character |
| crossing_subtype | character |
| crossing_fix | character |
| diameter_or_span_meters | numeric |
| length_or_width_meters | numeric |
| assessment_comment | character |
| rowid | numeric |
| date | Date |
| utm_zone | numeric |
| easting | numeric |
| northing | numeric |
| continuous_embeddedment_yes_no | character |
| average_depth_embededdment_meters | numeric |
| resemble_channel_yes_no | character |
| backwatered_yes_no | character |
| percentage_backwatered | numeric |
| fill_depth_meters | numeric |
| outlet_drop_meters | numeric |
| outlet_pool_depth_0_01m | numeric |
| inlet_drop_yes_no | character |
| culvert_slope_percent | numeric |
| downstream_channel_width_meters | numeric |
| stream_slope | numeric |
| beaver_activity_yes_no | character |
| fish_observed_yes_no | character |
| valley_fill | character |
| habitat_value | character |
| stream_width_ratio | numeric |
| culvert_length_score | numeric |
| embed_score | numeric |
| outlet_drop_score | numeric |
| culvert_slope_score | numeric |
| stream_width_ratio_score | numeric |
| final_score | numeric |
| barrier_result | character |
| recommended_diameter_or_span_meters | numeric |
| source | character |
| time_start | character |
| X.rowid | numeric |
| mergin_user | character |
| erosion_issues | integer |
| embankment_fill_issues | integer |
| blockage_issues | integer |
| condition_rank | integer |
| condition_notes | character |
| likelihood_flood_event_affecting_culvert | integer |
| consequence_flood_event_affecting_culvert | integer |
| climate_change_flood_risk | integer |
| vulnerability_rank | integer |
| climate_notes | character |
| traffic_volume | integer |
| community_access | integer |
| cost | integer |
| constructability | integer |
| fish_bearing | integer |
| environmental_impacts | integer |
| priority_rank | integer |
| overall_rank | integer |
| priority_notes | character |
| photo_road | character |
| photo_upstream | character |
| photo_downstream | character |
| photo_inlet | character |
| photo_barrel | character |
| photo_outlet | character |
| photo_condition | character |
| photo_embankment_fill | character |
| photo_blockage | character |
| photo_paper_card | character |
| photo_extra1 | character |
| photo_extra2 | character |
| photo_extra1_tag | character |
| photo_extra2_tag | character |
| link_method_phase1 | character |
| geom | sfc_POINT |
Let’s compare the differences:
identical(gpkp_q_sum, gpkp_r_sum)
## [1] FALSE
waldo::compare(gpkp_q_sum, gpkp_r_sum)
## `attr(old, 'row.names')[86:88]`: 86 87 88
## `attr(new, 'row.names')[86:89]`: 86 87 88 89
##
## old vs new
## name type
## - old[1, ] date_time_start DateTime
## + new[1, ] date_time_start POSIXct
## - old[2, ] pscis_crossing_id Real
## + new[2, ] pscis_crossing_id numeric
## - old[3, ] my_crossing_reference Integer
## + new[3, ] my_crossing_reference integer
## - old[4, ] crew_members String
## + new[4, ] crew_members character
## - old[5, ] aggregated_crossings_id Real
## + new[5, ] aggregated_crossings_id numeric
## - old[6, ] camera_id String
## + new[6, ] camera_id character
## - old[7, ] site_id Real
## + new[7, ] site_id numeric
## - old[8, ] gps_id String
## + new[8, ] gps_id character
## - old[9, ] moti_chris_culvert_id Integer
## + new[9, ] moti_chris_culvert_id integer
## - old[10, ] gps_waypoint_number String
## + new[10, ] gps_waypoint_number character
## and 79 more ...
##
## `old$name[86:88]`: "photo_extra1_tag" "photo_extra2_tag" "link_method_phase1"
## `new$name[86:89]`: "photo_extra1_tag" "photo_extra2_tag" "link_method_phase1" "geom"
##
## old$type | new$type
## [1] "DateTime" - "POSIXct" [1]
## [2] "Real" - "numeric" [2]
## [3] "Integer" - "integer" [3]
## [4] "String" - "character" [4]
## [5] "Real" - "numeric" [5]
## [6] "String" - "character" [6]
## [7] "Real" - "numeric" [7]
## [8] "String" - "character" [8]
## [9] "Integer" - "integer" [9]
## [10] "String" - "character" [10]
## ... ... ... and 79 more ...
Ok. Seems perhaps reasonable that we get some discontinuity with
modifications in R. We do have systems to track changes
with git and csvs. Clunky but they do work somewhat (I
think).
What about if we do something by hand? That should be fine and
geodiff should be able to track.
First let’s remove the project again and redownload to confirm our status is all good
system2("rm", args = c("-rf", "/Users/airvine/Projects/gis/py_client_error"))
# now we download
system2('conda', args = c('run', '-n', 'dff2', 'mergin download newgraph/py_client_error /Users/airvine/Projects/gis/py_client_error'), stdout = TRUE)
## [1] "Downloading into /Users/airvine/Projects/gis/py_client_error"
## [2] ""
## [3] "Done"
## [4] ""
# check status
result <- mergin_call()
## -\|/-\|/-\|/-\### Server changes:
## ### Local changes:
## ### Local changes summary ###
##
## |/ Exit status: 0
## Output:
## ### Server changes:
## ### Local changes:
## ### Local changes summary ###
We opened the project in QGIS and make a change to the
form_pscis table “by hand”. See screenshot.
# append scripts/qgis/mergin/ to the front would make snese...
knitr::include_graphics("fig/Screen Shot 2024-05-02 at 10.02.21 AM.png")
Then we checked the status again.
result <- mergin_call()
## -\|/-\|/-\|/-### Server changes:
## ### Local changes:
## ### Local changes summary ###
##
## \| Exit status: 0
## Output:
## ### Server changes:
## ### Local changes:
## ### Local changes summary ###
Ok. So I don’t think we have a problem with R and types.
As interesting as all that is - it seems that even a manual change in
the gpkg triggers an error. We will have to look into this further.
TO DO:
Test if altering the gpkg by hand throws error if the file was never
read into R….. There could be corruptions in the file history over time
that trigger error with geodiff every time there is a
change…..
py-api-client v0.8.3Let’s have a look at what happens if we try to check the status with
the py-api-client v0.8.3 version.
# Define the command and working directory
args <- c('run', '-n', 'dff', 'mergin', 'status')
result <- mergin_call()
## -\|/-\|/-\|/-Error: HTTP Error: 405 METHOD NOT ALLOWED
## URL: https://app.merginmaps.com/v1/project/py_client_error?since=v23
## Method: GET
## Detail: The method is not allowed for the requested URL.
##
## \| Exit status: 0
## Output:
## Error: HTTP Error: 405 METHOD NOT ALLOWED
## URL: https://app.merginmaps.com/v1/project/py_client_error?since=v23
## Method: GET
## Detail: The method is not allowed for the requested URL.
args <- c('run', '-n', 'dff', 'mergin', 'push')
result <- mergin_call()
## -\|/-\|/-\|/-Error: HTTP Error: 405 METHOD NOT ALLOWED
## URL: https://app.merginmaps.com/v1/project/py_client_error
## Method: GET
## Detail: The method is not allowed for the requested URL.
##
## \| Exit status: 0
## Output:
## Error: HTTP Error: 405 METHOD NOT ALLOWED
## URL: https://app.merginmaps.com/v1/project/py_client_error
## Method: GET
## Detail: The method is not allowed for the requested URL.
We can’t push!!
Looks like things have improved! Nice work ninjas!! ⚔️
Let’s try one last thing. We are going to download the file with the
QGIS plugin, make a change to the form_pscis table and then
check the status again.
First we remove the project.
# turned off - remove the project
system2("rm", args = c("-rf", "/Users/airvine/Projects/gis/py_client_error"))
Then we download the project with the QGIS plugin.
# Download the project with the QGIS plugin
# oddly seems we neeed to run from here vs with scripts/qgis/mergin/ at the front. don't get it
knitr::include_graphics("fig/Screen Shot 2024-05-02 at 12.42.11 AM.png")
Check the status with v0.9.0 again
args <- c('run', '-n', 'dff2', 'mergin', 'status')
result <- mergin_call()
## -\|/-\|/-\|/-\### Server changes:
## ### Local changes:
## ### Local changes summary ###
##
## |/ Exit status: 0
## Output:
## ### Server changes:
## ### Local changes:
## ### Local changes summary ###
Now we make a trivial change with R - same as before.
form_in <- sf::st_read(paste0(working_directory, "/form_pscis.gpkg"))
## Reading layer `form_pscis' from data source
## `/Users/airvine/Projects/gis/py_client_error/form_pscis.gpkg' using driver `GPKG'
## Simple feature collection with 16 features and 88 fields
## Geometry type: POINT
## Dimension: XY
## Bounding box: xmin: 948064.9 ymin: 447409.8 xmax: 1036232 ymax: 1000702
## Projected CRS: NAD83 / BC Albers
form_in |>
dplyr::mutate(my_crossing_reference = sample(1:10000, 1)) |>
sf::st_write(paste0(working_directory, "/form_pscis.gpkg"), delete_dsn = TRUE)
## Deleting source `/Users/airvine/Projects/gis/py_client_error/form_pscis.gpkg' using driver `GPKG'
## Writing layer `form_pscis' to data source
## `/Users/airvine/Projects/gis/py_client_error/form_pscis.gpkg' using driver `GPKG'
## Writing 16 features with 88 fields and geometry type Point.
Check the status again.
args <- c('run', '-n', 'dff2', 'mergin', 'status')
result <- mergin_call()
## -\|/-\|/-\|/-/var/folders/mg/h910y2c54fsc99qj74dyrjph0000gn/T/tmpoa97nbya: line 3: 93289 Segmentation fault: 11 mergin status
##
## ERROR conda.cli.main_run:execute(124): `conda run mergin status` failed. (See above for error)
## \| An error occurred: System command 'conda' failed
## Failed to execute the command properly.
This is the captured output from the terminal. There is more with
processx but that is not relevant I don’t think.
/var/folders/mg/h910y2c54fsc99qj74dyrjph0000gn/T/tmp6qsmgvr7: line 3: 82287 Segmentation fault: 11 mergin status
ERROR conda.cli.main_run:execute(124): `conda run mergin status` failed. (See above for error)
AHA!! It seemed like it was the QGIS plugin. But our results were inconsistent. Crazy. Not sure what is going on.
This is interesting too though.. We can still push and pull no problem.
args <- c('run', '-n', 'dff2', 'mergin', 'push')
result <- run(
command,
args = args,
echo = TRUE, # Print the command output live (similar to stdout = TRUE in system2)
wd = working_directory, # Set the working directory
spinner = TRUE, # Show a spinner in the R console while the command runs
timeout = 60 # Set a timeout for the command to complete
)
Let’s get the chunk ready to do our commits of these files to address our issue. We turn it off though.
# we need to derive the properpath to the file for git to see it
path_file_out <- paste0(
tools::file_path_sans_ext(params$file_in),
'.html'
)
gert::git_add(
c(params$file_in, path_file_out)
)
message <- "initial commit file to address #166"
message <- "update file to address #166 and include git commits"
gert::git_commit(message)
gert::git_push()
We can read our docs online so lets add a comment in the issue that points to the file:
# construct the url of the viewer
url_viewer <- paste0(
"http://htmlpreview.github.io/?",
params$repo_url,
"blob/",
params$repo_branch,
"/",
path_file_out
)
gh::gh(endpoint = "POST /repos/{owner}/{repo}/issues/{issue_number}/comments",
body = paste0("The documentation for this is available here ", url_viewer, " ."),
owner = params$repo_owner,
repo = params$repo_name,
issue_number = 166
)
## {
## "url": "https://api.github.com/repos/NewGraphEnvironment/dff-2022/issues/comments/2091156922",
## "html_url": "https://github.com/NewGraphEnvironment/dff-2022/issues/166#issuecomment-2091156922",
## "issue_url": "https://api.github.com/repos/NewGraphEnvironment/dff-2022/issues/166",
## "id": 2091156922,
## "node_id": "IC_kwDOHyEXG858pIW6",
## "user": {
## "login": "NewGraphEnvironment",
## "id": 46538103,
## "node_id": "MDQ6VXNlcjQ2NTM4MTAz",
## "avatar_url": "https://avatars.githubusercontent.com/u/46538103?u=aaf04ae3fc7d68a0dec3202562cedcf27ce188a8&v=4",
## "gravatar_id": "",
## "url": "https://api.github.com/users/NewGraphEnvironment",
## "html_url": "https://github.com/NewGraphEnvironment",
## "followers_url": "https://api.github.com/users/NewGraphEnvironment/followers",
## "following_url": "https://api.github.com/users/NewGraphEnvironment/following{/other_user}",
## "gists_url": "https://api.github.com/users/NewGraphEnvironment/gists{/gist_id}",
## "starred_url": "https://api.github.com/users/NewGraphEnvironment/starred{/owner}{/repo}",
## "subscriptions_url": "https://api.github.com/users/NewGraphEnvironment/subscriptions",
## "organizations_url": "https://api.github.com/users/NewGraphEnvironment/orgs",
## "repos_url": "https://api.github.com/users/NewGraphEnvironment/repos",
## "events_url": "https://api.github.com/users/NewGraphEnvironment/events{/privacy}",
## "received_events_url": "https://api.github.com/users/NewGraphEnvironment/received_events",
## "type": "User",
## "site_admin": false
## },
## "created_at": "2024-05-02T17:45:36Z",
## "updated_at": "2024-05-02T17:45:36Z",
## "author_association": "OWNER",
## "body": "The documentation for this is available here http://htmlpreview.github.io/?https://github.com/NewGraphEnvironment/dff-2022/blob/master/scripts/qgis/mergin/py_client_error.html .",
## "reactions": {
## "url": "https://api.github.com/repos/NewGraphEnvironment/dff-2022/issues/comments/2091156922/reactions",
## "total_count": 0,
## "+1": 0,
## "-1": 0,
## "laugh": 0,
## "hooray": 0,
## "confused": 0,
## "heart": 0,
## "rocket": 0,
## "eyes": 0
## },
## "performed_via_github_app": {}
## }
rmarkdown::render(input = params$file_in, output_file = params$file_out)